# -*- coding: utf-8 -*-
import requests  # 导入requests
import random  # 取随机数
import time
import re
from lxml import etree
from db.mysql_conn import MysqlUtils, config, sqlStr
mysql = MysqlUtils(config)
from config.conf import proxiesDEF, Referer_list
from config.useragent import user_agent_list
get_goods_num = 0
goodsArr = []
user_agent_len = len(user_agent_list) - 10
Referer_list_len = len(Referer_list) - 1

def getAsin():
    try:
        # goodsArr = mysql.query('select id, dp from book where dp = "1119100461" limit 1')
        goodsArr = mysql.query('select id, dp from book where price = 0 and id > 139500 limit 50')
    except:
        goodsArr = []
    proxies_list = ['221.122.91.60:80', '125.123.127.251:9000', '125.73.220.18:49128', '106.12.201.174:8080', '125.123.124.192:9000', '125.123.126.137:9000', '125.123.65.5:9000', '27.195.216.21:8118', '222.128.9.235:33428', '125.123.121.214:9000', '114.249.113.151:9000', '123.56.74.221:80', '223.156.112.100:9000', '47.104.172.108:8118', '125.123.124.145:9000', '221.122.91.65:80', '122.136.212.132:53281', '125.123.44.147:9000', '218.108.175.15:80', '52.83.202.3:8118', '47.107.158.219:8000', '124.205.155.157:9090', '182.35.81.10:9999', '45.125.32.181:3128', '182.35.86.175:9999', '1.198.72.162:9999', '163.204.242.30:9999', '125.123.44.81:9000', '61.160.210.223:808', '119.178.101.18:8888', '221.122.91.61:80', '123.132.232.254:61017', '114.225.29.112:8118', '43.254.168.56:53281', '69.160.173.3:8000', '123.163.97.154:9999', '163.204.244.166:9999', '163.204.243.219:9999', '163.204.245.109:9999', '125.123.66.161:9000', '134.175.55.15:8888', '182.35.80.251:9999', '125.123.141.95:9000', '163.125.70.239:8888', '218.60.8.83:3129', '125.65.79.60:3311', '116.52.100.233:8888', '182.122.205.72:8118', '106.9.169.41:808', '175.43.34.17:9999']

    # boo = [True, False, True, False, False, False, False, False, False]
    boo = [True]
    try:
        if random.choice(boo):
            proxies_list = proxiesDEF()
            print(proxies_list)
    except:
        print('获取ip失败')
    if len(goodsArr) > 0:
        getGoodsDeDetailTwo(goodsArr, proxies_list)
    else:
        print('=========over==============')


def getGoodsDeDetailTwo (goodsArr, proxies_list):
    global get_goods_num
    for index, goods in enumerate(goodsArr):
        print('==%s==================%s=================================' % (goods['id'], get_goods_num))
        get_goods_num = get_goods_num + 1
        # if get_goods_num % 300 == 0:
        #     print('休息一会==========马上开始')
        #     time.sleep(100 + random.uniform(13, 66))
        url = 'https://www.amazon.com/dp/' + goods['dp']
        print(url)
        proxies = {'http': proxies_list[random.randint(0, len(proxies_list)-1)]}
        headers = {
            'User-Agent': user_agent_list[random.randint(0, user_agent_len)],
            'Referer': Referer_list[random.randint(0, Referer_list_len)]
        }
        try:
            session = requests.session()
            response = session.get(url, headers=headers, proxies=proxies, timeout=(3, 7))
            response_soup = etree.HTML(response.content.decode('utf-8'))
            time.sleep(random.uniform(0.5, 0.7))
            # htmlstr = response.content.decode('utf-8')
            try:
                name = response_soup.xpath('//*[@id="productTitle"]/text()')[0].strip()
            except:
                name = "xxxxxxxxx"
            try:
                resArr = response_soup.xpath('//li[@id="SalesRank"]/text()')
                str = ','.join(resArr)
                restr = re.compile('#(.*) \(,\)')
                rankarr = restr.findall(str)[0].split(' in ')
                ranking = rankarr[0].replace(',', '')
                rankingtype = rankarr[1]
            except:
                ranking = -1
                rankingtype = ''
            try:
                ISBN10 = response_soup.xpath('//div/span[contains(text(), "ISBN-10:")]/following-sibling::span[1]/text()')[0].strip()
            except:
                ISBN10 = ''
            try:
                ISBN13 = response_soup.xpath('//div/span[contains(text(), "ISBN-13:")]/following-sibling::span[1]/text()')[0].strip()
            except:
                ISBN13 = ''
            try:
                asin = response_soup.xpath('//input[@id="ASIN"]/@value')[0].strip()
            except:
                asin = ''
            try:
                Paperback = response_soup.xpath('//div[@class="a-row"]/span[contains(text(), "Paperback")]/../following-sibling::div[1]/span/text()')[0].strip()
            except:
                Paperback = ''
            try:
                Hardcover = response_soup.xpath('//div[@class="a-row"]/span[contains(text(), "Hardcover")]/../following-sibling::div[1]/span/text()')[0].strip()
            except:
                Hardcover = ''

            goods['ranking'] = ranking
            goods['rankingtype'] = rankingtype
            goods['ISBN10'] = ISBN10
            goods['ISBN13'] = ISBN13
            goods['asin'] = asin
            print(name)
            print('Paperback ==== %s  ' % (Paperback))
            print(Paperback)
            print('Hardcover ==== %s  ' % (Hardcover))
            print(Hardcover)
            if Paperback != '' and Hardcover != '':
                try:
                    sql = 'select id, dp, star, reviews, url, name, img, country from book where id= %s' % (goods['id'])
                    deleteGoods = mysql.query(sql)[0]
                    if Paperback != '':
                        try:
                            goods['packagetype'] = 'Paperback'
                            Paperback = Paperback.replace('from', '').strip()
                            moneytype = Paperback[0:1]
                            goods['moneytype'] = moneytype
                            goods['price'] = averageNum(Paperback.replace(moneytype, ''))
                            sqlParam = returnGoods(goods, deleteGoods)
                            mysql.insert(sqlStr, sqlParam)
                            print('double insert paperback')
                        except:
                            print('Paperback and Hardcover   Paperback=== error')

                    if Hardcover != '':
                        try:
                            goods['packagetype'] = 'Hardcover'
                            Hardcover = Hardcover.replace('from', '').strip()
                            moneytype = Hardcover[0:1]
                            goods['moneytype'] = moneytype
                            goods['price'] = averageNum(Hardcover.replace(moneytype, ''))
                            sqlParam = returnGoods(goods, deleteGoods)
                            mysql.insert(sqlStr, sqlParam)
                            print('double insert Hardcover')
                        except:
                            print('Paperback and Hardcover   Hardcover=== error')

                    deletesql = "DELETE FROM book WHERE id =  %s" % (goods['id'])
                    mysql.delete(deletesql)
                except:
                    print('Paperback and Hardcover   === error')
            else:
                if Paperback != '':
                    try:
                        goods['packagetype'] = 'Paperback'
                        Paperback = Paperback.replace('from', '').strip()
                        moneytype = Paperback[0:1]
                        goods['moneytype'] = moneytype
                        goods['price'] = averageNum(Paperback.replace(moneytype, ''))
                        print(goods)
                        updateSql(goods)
                        print('Paperback   update')
                    except:
                        print('Paperback   err')
                if Hardcover != '':
                    try:
                        goods['packagetype'] = 'Hardcover'
                        Hardcover = Hardcover.replace('from', '').strip()
                        moneytype = Hardcover[0:1]
                        goods['moneytype'] = moneytype
                        goods['price'] = averageNum(Hardcover.replace(moneytype, ''))
                        print(goods)
                        updateSql(goods)
                        print('Hardcover   update')
                    except Exception as exc:
                        print(exc)
                        print('Hardcover   err')
                if Paperback == '' and Hardcover == '' and name !='xxxxxxxxx':
                    try:
                        goods['packagetype'] = 'other'
                        goods['moneytype'] = ''
                        goods['price'] = -1
                        print(goods)
                        updateSql(goods)
                        print('packagetype Hardcover xxxxxxxxx updateSql')
                    except Exception as exc:
                        print(exc)
                        print('packagetype Hardcover xxxxxxxxx  err')

            # 清空数组 goods
            print('*goods***********************************')
            goods.clear()
            print(goods)

        except Exception as exc:
            print(exc)
            print('get 请求失败 跳往下一个')
    getAsin()


def updateSql(goods):
    d = {
        'packagetype': goods['packagetype'],
        'ISBN10': goods['ISBN10'],
        'ISBN13': goods['ISBN13'],
        'asin': goods['asin'],
        'rankingtype': goods['rankingtype'],
        'ranking': goods['ranking'],
        'moneytype': goods['moneytype'],
        'price': goods['price'],
         }  # 更新的字段及对应值
    t = ('id', goods['id'])  # 更新所需条件
    table_name = 'book'
    updateSqlStr = 'UPDATE %s SET ' % table_name + ','.join(['%s=%r' % (k, d[k]) for k in d]) + ',updateTime = NOW() WHERE %s=%r;' % (t[0], t[1])
    mysql.update(updateSqlStr)


def returnGoods(goods, deleteGoods):
    sqlParam = [
        goods['dp'],
        goods['packagetype'],
        '',
        goods['ISBN10'],
        goods['ISBN13'],
        goods['asin'],
        goods['rankingtype'],
        goods['ranking'],
        deleteGoods['star'],
        deleteGoods['reviews'],
        goods['moneytype'],
        goods['price'],
        deleteGoods['url'],
        deleteGoods['name'],
        deleteGoods['img'],
        deleteGoods['country']
    ]
    return sqlParam

def averageNum(str):
    try:
        if ' - ' in str:
            arr = str.split(' - ')
            add = float(arr[0]) + float(arr[1])
            price = '%.2f' % float(add/2)
        else:
            price = '%.2f' % float(str)
    except:
        price = -1
    return price

if __name__ == '__main__':
    getAsin()




